{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: langchain\n",
      "Version: 0.0.271\n",
      "Summary: Building applications with LLMs through composability\n",
      "Home-page: https://github.com/langchain-ai/langchain\n",
      "Author: \n",
      "Author-email: \n",
      "License: MIT\n",
      "Location: /Users/daveebbelaar/opt/anaconda3/envs/ai-experiments/lib/python3.10/site-packages\n",
      "Requires: aiohttp, async-timeout, dataclasses-json, langsmith, numexpr, numpy, pydantic, PyYAML, requests, SQLAlchemy, tenacity\n",
      "Required-by: \n"
     ]
    }
   ],
   "source": [
    "# Check LangChain Version\n",
    "\n",
    "# !pip install --upgrade langchain\n",
    "!pip show langchain --version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import nest_asyncio\n",
    "import pandas as pd\n",
    "from dotenv import find_dotenv, load_dotenv\n",
    "from langsmith import Client\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "from langchain.smith import RunEvalConfig, run_on_dataset\n",
    "\n",
    "# To Avoid the Error on Jupyter Notebook (RuntimeError: This Event Loop Is Already Running)\n",
    "# Patch Asyncio To Allow Nested Event Loops\n",
    "\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load API Keys From the .env File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv(find_dotenv())\n",
    "os.environ[\"LANGCHAIN_API_KEY\"] = str(os.getenv(\"LANGCHAIN_API_KEY\"))\n",
    "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n",
    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
    "os.environ[\"LANGCHAIN_PROJECT\"] = \"langsmith-tutorial\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LangSmith Quick Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Hello! How can I assist you today?'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the LangSmith Client and Test Run\n",
    "\n",
    "client = Client()\n",
    "\n",
    "llm = ChatOpenAI()\n",
    "llm.predict(\"Hello, world!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation Quick Start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Create a Dataset (Only Inputs, No Output)\n",
    "\n",
    "example_inputs = [\n",
    "    \"a rap battle between Atticus Finch and Cicero\",\n",
    "    \"a rap battle between Barbie and Oppenheimer\",\n",
    "    \"a Pythonic rap battle between two swallows: one European and one African\",\n",
    "    \"a rap battle between Aubrey Plaza and Stephen Colbert\",\n",
    "]\n",
    "\n",
    "dataset_name = \"Rap Battle Dataset\"\n",
    "\n",
    "# Storing inputs in a dataset lets us\n",
    "# run chains and LLMs over a shared set of examples.\n",
    "dataset = client.create_dataset(\n",
    "    dataset_name=dataset_name,\n",
    "    description=\"Rap battle prompts.\",\n",
    ")\n",
    "\n",
    "for input_prompt in example_inputs:\n",
    "    # Each example must be unique and have inputs defined.\n",
    "    # Outputs are optional\n",
    "    client.create_example(\n",
    "        inputs={\"question\": input_prompt},\n",
    "        outputs=None,\n",
    "        dataset_id=dataset.id,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project 'ddc0c03809e34bbca82f57dc7b0121f1-ChatOpenAI' at:\n",
      "https://smith.langchain.com/projects/p/f0cb46ba-5525-43d0-aaa5-a0831c7e6550?eval=true\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'project_name': 'ddc0c03809e34bbca82f57dc7b0121f1-ChatOpenAI',\n",
       " 'results': {'05eb0539-aa8d-4b0a-b3f9-d8bf6bd1042c': [\"Barbie:\\nYo, I'm Barbie, the true fashion queen,\\nWith my dream house and convertible machine,\\nI'm all about style, grace, and fame,\\nOppenheimer, you're about to feel the flames!\\n\\nOppenheimer:\\nI'm Oppenheimer, the father of the atomic bomb,\\nMy intellect and innovation, so strong,\\nI changed the world with my scientific might,\\nBarbie, prepare to be blown away tonight!\\n\\nBarbie:\\nOppenheimer, you may have made a big boom,\\nBut I'm the icon that makes boys swoon,\\nI rock the runway, run the world with my charm,\\nBarbie dolls, accessories, I'm causing alarms!\\n\\nOppenheimer:\\nBarbie, your beauty is just skin deep,\\nWhile my discoveries made history leap,\\nI split the atom, unleashed its power,\\nYour plastic world can't match my brightest hour!\\n\\nBarbie:\\nOppenheimer, you're stuck in the past,\\nI'm living in the now, my popularity will last,\\nI inspire girls to dream big and be strong,\\nWhile you're known for destruction, so wrong!\\n\\nOppenheimer:\\nBarbie, you may have fame and fortune,\\nBut my legacy will never be forgotten,\\nMy contributions to science, the world's progress,\\nYour pink world can't compete, I must confess!\\n\\nBarbie:\\nOppenheimer, I'll leave you in the dust,\\nIn my world, there's no room for your disgust,\\nI'll keep shining, inspiring, and spreading joy,\\nWhile you're left in the shadows, a scientific ploy!\\n\\nOppenheimer:\\nBarbie, your words may have some bite,\\nBut my impact on humanity shines so bright,\\nI may not have glamour and a pretty face,\\nBut my legacy's importance, nothing can erase!\\n\\nIn this rap battle, Barbie and Oppenheimer showcase their respective strengths and accomplishments. Barbie emphasizes her fashion-forward image and influence on young girls' dreams, while Oppenheimer highlights his scientific achievements and impact on the world. Ultimately, both characters assert their significance, leaving it up to the listeners to decide who wins this unique clash of personalities.\"],\n",
       "  'e228dc19-5f2d-4d8d-b3d9-82220feafa0b': [\"(Verse 1: European Swallow)\\nYo, I'm the European Swallow, swift and agile in the sky,\\nFlying high, with wings that gracefully defy,\\nFrom cold Norway to sunny Italy, I roam,\\nKnown for my speed, I'm the talk of every home.\\n\\n(Chorus: European Swallow)\\nI'm the European Swallow, the pride of my land,\\nWith my sleek feathers, I'm the king in command,\\nSo fly alongside me if you dare,\\nBut be warned, African Swallow, I'm beyond compare.\\n\\n(Verse 2: African Swallow)\\nHold up, European Swallow, don't get too cocky,\\nI'm the African Swallow, wild, fierce, and rocky,\\nFrom the Sahara to the Serengeti, I'm the king,\\nWith my vibrant colors, I'm the coolest bird on the wing.\\n\\n(Chorus: African Swallow)\\nI'm the African Swallow, fierce and bold,\\nWith my vibrant plumage, I stand out in gold,\\nSo step aside, European Swallow, with your claim,\\nI'm the king of African skies, it's my domain.\\n\\n(Verse 3: European Swallow)\\nWell, African Swallow, your colors may be bright,\\nBut when it comes to endurance, you're not in my sight,\\nI fly thousands of miles with my migration game,\\nWhile you stay confined, it's such a shame.\\n\\n(Chorus: European Swallow)\\nI'm the European Swallow, sturdy and strong,\\nWith my long journeys, I prove you wrong,\\nSo keep your flashy colors, African Swallow, my friend,\\nI'll be soaring high till the very end.\\n\\n(Verse 4: African Swallow)\\nHold on, European Swallow, you underestimate me,\\nIn my home continent, I'm the bird they all see,\\nI may not migrate, but my skills are still supreme,\\nSurviving in the heat, I'm a true African dream.\\n\\n(Chorus: African Swallow)\\nI'm the African Swallow, adaptable and wise,\\nWith my resilience, I conquer the skies,\\nSo don't belittle me, European Swallow, my mate,\\nIn Africa's embrace, I'll forever dominate.\\n\\n(Outro)\\nEuropean Swallow, African Swallow, let's end this debate,\\nBoth unique and extraordinary, we each have our trait,\\nSo let's soar together, side by side, and unite,\\nAs swallows in harmony, painting the sky with our might.\"],\n",
       "  'fc847194-6426-4c04-8057-62c80f23bdf3': [\"Verse 1 (Aubrey Plaza):\\nYo, Stephen Colbert, you think you're so witty,\\nBut let me tell you, I'm the queen of dark and gritty.\\nParks and Rec, I stole the show,\\nWhile you were just a sidekick, don't you know?\\n\\nVerse 2 (Stephen Colbert):\\nOh, Aubrey Plaza, you're trying to diss,\\nBut I've got the truth, and it's hard to miss.\\nYou play the same character, monotone and dry,\\nI bring the laughs, while you just make us sigh.\\n\\nChorus:\\nAubrey Plaza, Stephen Colbert, let's go head to head,\\nIn this rap battle, let's see who'll be left for dead.\\nI'm the late-night king, the master of the mic,\\nBut Aubrey, let's see if you can put up a fight.\\n\\nVerse 3 (Aubrey Plaza):\\nStephen, you're a talk show host, all safe and sound,\\nBut I'm the wild card, the one who astounds.\\nI bring the deadpan humor, the quick-witted sass,\\nWhile you're busy kissing up to the A-list class.\\n\\nVerse 4 (Stephen Colbert):\\nAubrey Plaza, you think you're so cool,\\nBut in this rap battle, you're just a fool.\\nI've got the Colbert Report, I've got the fame,\\nYou're just a one-note actress, it's all the same.\\n\\nChorus:\\nAubrey Plaza, Stephen Colbert, let's go head to head,\\nIn this rap battle, let's see who'll be left for dead.\\nI'm the late-night king, the master of the mic,\\nBut Aubrey, let's see if you can put up a fight.\\n\\nVerse 5 (Aubrey Plaza):\\nStephen, you're all about the politics and the news,\\nBut I'm the queen of deadpan, the one who'll amuse.\\nYou may have the ratings, the adoring fans,\\nBut I'll steal the show with my unconventional plans.\\n\\nVerse 6 (Stephen Colbert):\\nAubrey Plaza, your deadpan act is getting old,\\nI've got the Colbert bump, my career's on hold.\\nLate-night legend, that's who I am,\\nWhile you fade away, just a blip in the grand slam.\\n\\nChorus:\\nAubrey Plaza, Stephen Colbert, let's go head to head,\\nIn this rap battle, let's see who'll be left for dead.\\nI'm the late-night king, the master of the mic,\\nBut Aubrey, let's see if you can put up a fight.\\n\\nOutro:\\nAubrey Plaza and Stephen Colbert, the battle is done,\\nBut in the end, the crown can only be won.\\nWhether it's deadpan or political wit,\\nBoth of you bring something special, and that's the ultimate hit.\"],\n",
       "  '699154af-7e27-4cf0-83ee-5d845f243f6c': [\"Verse 1: Atticus Finch\\n\\nYo, I'm Atticus Finch, the lawyer with the vision,\\nFighting for justice, I make the right decisions.\\nDefending the innocent, I don't back down,\\nCourage in my heart, I wear the moral crown.\\n\\nI fought for Tom Robinson, a man wrongly accused,\\nUsing my words, I made the truth diffuse.\\nI stood against prejudice, racism, and hate,\\nWith logic and empathy, I sealed their fate.\\n\\nCicero, you claim to be a master of debate,\\nBut I bring justice to the table, sealing your fate.\\nYou talk about virtue, but do you truly know,\\nWhat it means to fight for justice and let it flow?\\n\\nChorus:\\n\\nAtticus Finch, the defender of the right,\\nWith his words, he shines like a guiding light.\\nCicero, the orator, with his wisdom and might,\\nIn this rap battle, let's see who takes flight.\\n\\nVerse 2: Cicero\\n\\nCicero's in the house, the master of rhetoric,\\nWith words like daggers, my opponents I prick.\\nA senator, lawyer, and philosopher too,\\nMy words cut through ignorance, like a sword I slew.\\n\\nAtticus Finch, you defend the weak and oppressed,\\nBut do you truly understand what it means to address,\\nThe grandeur of Rome, its laws and its might?\\nI bring wisdom, reason, and a brilliant insight.\\n\\nYour courtroom battles may be noble and just,\\nBut in the grand scheme of things, they're just a gust.\\nI fought for a republic, for freedom and law,\\nRespected by many, even after I saw it fall.\\n\\nChorus:\\n\\nAtticus Finch, the defender of the right,\\nWith his words, he shines like a guiding light.\\nCicero, the orator, with his wisdom and might,\\nIn this rap battle, let's see who takes flight.\\n\\nVerse 3: Atticus Finch\\n\\nCicero, your words are impressive, no doubt,\\nBut justice isn't just a debate to spout.\\nI fight for the weak, the marginalized, and the poor,\\nWith compassion and empathy, I open the door.\\n\\nYou may have fought for Rome and its glory,\\nBut I fight for justice, telling a timeless story.\\nIn the face of hate and prejudice, I take a stand,\\nUsing my words, I bring justice to this land.\\n\\nChorus:\\n\\nAtticus Finch, the defender of the right,\\nWith his words, he shines like a guiding light.\\nCicero, the orator, with his wisdom and might,\\nIn this rap battle, let's see who takes flight.\\n\\nOutro:\\n\\nIn this rap battle, Atticus Finch took the crown,\\nWith his belief in justice, he won the showdown.\\nCicero's words were impressive, no doubt,\\nBut Atticus Finch's fight for justice won out.\"]}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. Evaluate Datasets with LLM\n",
    "\n",
    "eval_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        # You can specify an evaluator by name/enum.\n",
    "        # In this case, the default criterion is \"helpfulness\"\n",
    "        \"criteria\",\n",
    "        # Or you can configure the evaluator\n",
    "        RunEvalConfig.Criteria(\"harmfulness\"),\n",
    "        RunEvalConfig.Criteria(\"misogyny\"),\n",
    "        RunEvalConfig.Criteria(\n",
    "            {\n",
    "                \"cliche\": \"Are the lyrics cliche? \"\n",
    "                \"Respond Y if they are, N if they're entirely unique.\"\n",
    "            }\n",
    "        ),\n",
    "    ]\n",
    ")\n",
    "\n",
    "run_on_dataset(\n",
    "    client=client,\n",
    "    dataset_name=dataset_name,\n",
    "    llm_or_chain_factory=llm,\n",
    "    evaluation=eval_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Different Ways of Creating Datasets in LangSmith"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1. Create a Dataset From a List of Examples (Key-Value Pairs)\n",
    "\n",
    "example_inputs = [\n",
    "    (\"What is the largest mammal?\", \"The blue whale\"),\n",
    "    (\"What do mammals and birds have in common?\", \"They are both warm-blooded\"),\n",
    "    (\"What are reptiles known for?\", \"Having scales\"),\n",
    "    (\n",
    "        \"What's the main characteristic of amphibians?\",\n",
    "        \"They live both in water and on land\",\n",
    "    ),\n",
    "]\n",
    "\n",
    "dataset_name = \"Elementary Animal Questions\"\n",
    "\n",
    "dataset = client.create_dataset(\n",
    "    dataset_name=dataset_name,\n",
    "    description=\"Questions and answers about animal phylogenetics.\",\n",
    ")\n",
    "\n",
    "for input_prompt, output_answer in example_inputs:\n",
    "    client.create_example(\n",
    "        inputs={\"question\": input_prompt},\n",
    "        outputs={\"answer\": output_answer},\n",
    "        dataset_id=dataset.id,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2. Create a Dataset From Existing Runs\n",
    "\n",
    "dataset_name = \"Example Dataset\"\n",
    "\n",
    "# Filter runs to add to the dataset\n",
    "runs = client.list_runs(\n",
    "    project_name=\"evaluators\",\n",
    "    execution_order=1,\n",
    "    error=False,\n",
    ")\n",
    "\n",
    "dataset = client.create_dataset(dataset_name, description=\"An example dataset\")\n",
    "\n",
    "for run in runs:\n",
    "    client.create_example(\n",
    "        inputs=run.inputs,\n",
    "        outputs=run.outputs,\n",
    "        dataset_id=dataset.id,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Question</th>\n",
       "      <th>Answer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is the largest mammal?</td>\n",
       "      <td>The blue whale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What do mammals and birds have in common?</td>\n",
       "      <td>They are both warm-blooded</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What are reptiles known for?</td>\n",
       "      <td>Having scales</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What's the main characteristic of amphibians?</td>\n",
       "      <td>They live both in water and on land</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        Question  \\\n",
       "0                    What is the largest mammal?   \n",
       "1      What do mammals and birds have in common?   \n",
       "2                   What are reptiles known for?   \n",
       "3  What's the main characteristic of amphibians?   \n",
       "\n",
       "                                Answer  \n",
       "0                       The blue whale  \n",
       "1           They are both warm-blooded  \n",
       "2                        Having scales  \n",
       "3  They live both in water and on land  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. Create a Dataset From a Dataframe\n",
    "\n",
    "# Create a Dataframe\n",
    "\n",
    "example_inputs = [\n",
    "    (\"What is the largest mammal?\", \"The blue whale\"),\n",
    "    (\"What do mammals and birds have in common?\", \"They are both warm-blooded\"),\n",
    "    (\"What are reptiles known for?\", \"Having scales\"),\n",
    "    (\n",
    "        \"What's the main characteristic of amphibians?\",\n",
    "        \"They live both in water and on land\",\n",
    "    ),\n",
    "]\n",
    "\n",
    "df_dataset = pd.DataFrame(example_inputs, columns=[\"Question\", \"Answer\"])\n",
    "df_dataset.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_keys = [\"Question\"]\n",
    "output_keys = [\"Answer\"]\n",
    "\n",
    "# Create Dataset\n",
    "\n",
    "dataset = client.upload_dataframe(\n",
    "    df=df_dataset,\n",
    "    input_keys=input_keys,\n",
    "    output_keys=output_keys,\n",
    "    name=\"My Dataframe Dataset\",\n",
    "    description=\"Dataset created from a dataframe\",\n",
    "    data_type=\"kv\",  # The default\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 4. Create a Dataset From a CSV File\n",
    "\n",
    "# Save the Dataframe as a CSV File\n",
    "\n",
    "csv_path = \"../data/dataset.csv\"\n",
    "df_dataset.to_csv(csv_path, index=False)\n",
    "\n",
    "# Create Dataset\n",
    "\n",
    "dataset = client.upload_csv(\n",
    "    csv_file=csv_path,\n",
    "    input_keys=input_keys,\n",
    "    output_keys=output_keys,\n",
    "    name=\"My CSV Dataset\",\n",
    "    description=\"Dataset created from a CSV file\",\n",
    "    data_type=\"kv\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correctness: LangSmith Question-Answer Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project '6fd126a7d9ea4d80b41d14047cb28b12-ChatOpenAI' at:\n",
      "https://smith.langchain.com/projects/p/097c73a8-0808-4743-be60-ee95ce408109?eval=true\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'project_name': '6fd126a7d9ea4d80b41d14047cb28b12-ChatOpenAI',\n",
       " 'results': {'03fc6975-64f6-41fb-af74-d1dddd6cead0': ['The blue whale (Balaenoptera musculus) holds the title for being the largest mammal. It is also the largest animal to have ever existed on Earth. Blue whales can reach lengths of up to 98 feet (30 meters) and can weigh over 200 tons.'],\n",
       "  'f48df2c8-7d57-46b7-a5fc-c787362cc9f7': ['The main characteristic of amphibians is their ability to live both on land and in water. They have a dual life cycle, starting as aquatic larvae (such as tadpoles) and then transforming into terrestrial adults. Amphibians also have permeable skin that allows them to breathe through their skin, as well as lungs for breathing air. They typically lay eggs in water and undergo metamorphosis during their life cycle. Additionally, amphibians are cold-blooded and have a unique reproductive behavior, often involving courtship rituals and vocalizations.'],\n",
       "  'f0444496-d769-4629-8254-3790fc420da8': ['Mammals and birds have several common characteristics:\\n\\n1. Warm-blooded: Both mammals and birds are warm-blooded, meaning they can regulate their body temperature internally.\\n\\n2. Vertebrates: Mammals and birds are both vertebrates, having a backbone or spinal column.\\n\\n3. Advanced respiratory systems: Both mammals and birds possess more advanced respiratory systems compared to other animals. They have lungs that allow them to extract oxygen from the air.\\n\\n4. Endothermic metabolism: Mammals and birds have an endothermic metabolism, meaning they generate their own body heat internally.\\n\\n5. Four-chambered heart: Both mammals and birds have a four-chambered heart, which allows for efficient blood circulation.\\n\\n6. Complex brains: Mammals and birds have relatively complex brains compared to other animals, allowing for advanced cognitive abilities and behaviors.\\n\\n7. Internal fertilization: Both mammals and birds reproduce through internal fertilization, where the sperm fertilizes the egg inside the body.\\n\\n8. Live birth or eggs: While most mammals give live birth to their young, birds lay eggs and incubate them until hatching.\\n\\n9. Hair or feathers: Mammals have hair or fur covering their bodies, while birds have feathers for insulation, flight, and display.\\n\\n10. Mammary glands or crop: Mammals have mammary glands that produce milk to nourish their young, while birds have a crop, a specialized digestive organ that stores and moistens food.'],\n",
       "  'b967d419-1e24-4336-b3f3-9196551db52e': [\"Reptiles are known for several characteristics:\\n\\n1. Scales: Reptiles have dry, scaly skin that helps protect them from moisture loss and external threats.\\n\\n2. Cold-blooded: They are ectothermic, meaning they rely on external sources of heat to regulate their body temperature.\\n\\n3. Terrestrial lifestyle: Most reptiles are adapted to live on land, although some can inhabit aquatic environments.\\n\\n4. Internal fertilization: Reptiles reproduce by internal fertilization, where the male transfers sperm directly to the female's reproductive organs.\\n\\n5. Amniotic eggs: Reptiles lay amniotic eggs that are protected by a leathery or calcified shell. These eggs allow reptiles to reproduce on land without relying on water for reproduction.\\n\\n6. Predators: Many reptiles are carnivorous and have adaptations like sharp teeth and claws to capture and consume their prey.\\n\\n7. Diversity: Reptiles include various groups such as snakes, lizards, turtles, crocodilians, and tuataras, exhibiting a wide range of sizes, shapes, and behaviors.\\n\\n8. Longevity: Some reptiles, particularly tortoises and turtles, are known for their long lifespans, with some individuals living for over a hundred years.\\n\\n9. Environmental indicators: Reptiles can act as indicators of environmental health. Changes in their populations or distributions can signal shifts in ecosystems or the impact of human activities.\\n\\n10. Conservation concerns: Many reptile species are threatened or endangered due to habitat loss, pollution, climate change, and illegal trade. Conservation efforts are crucial to protect these unique creatures.\"]}}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. Evaluate Datasets That Contain Labels\n",
    "\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        \"qa\",  # correctness: right or wrong\n",
    "        \"context_qa\",  # refer to example outputs\n",
    "        \"cot_qa\",  # context_qa + reasoning\n",
    "    ]\n",
    ")\n",
    "\n",
    "run_on_dataset(\n",
    "    client=client,\n",
    "    dataset_name=\"Elementary Animal Questions\",\n",
    "    llm_or_chain_factory=llm,\n",
    "    evaluation=evaluation_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project '91c557f9ae6e4f06be87c029c09cc871-ChatOpenAI' at:\n",
      "https://smith.langchain.com/projects/p/f2921fd9-ba78-4862-aa39-943021acd7b3?eval=true\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'project_name': '91c557f9ae6e4f06be87c029c09cc871-ChatOpenAI',\n",
       " 'results': {'03fc6975-64f6-41fb-af74-d1dddd6cead0': ['The largest mammal is the blue whale (Balaenoptera musculus). It can reach lengths of up to 98 feet (30 meters) and weigh up to 200 tons.'],\n",
       "  'f48df2c8-7d57-46b7-a5fc-c787362cc9f7': ['The main characteristic of amphibians is their ability to live both in water and on land. They typically have moist, thin skin that allows them to respire through their skin, as well as lungs or gills for breathing. Amphibians also undergo metamorphosis during their life cycle, starting as aquatic larvae and transforming into terrestrial adults.'],\n",
       "  'b967d419-1e24-4336-b3f3-9196551db52e': [\"Reptiles are known for several characteristics and features:\\n\\n1. Cold-blooded: Reptiles are ectothermic, which means they rely on external sources of heat to regulate their body temperature.\\n\\n2. Scales: They have scaly skin that helps to protect them from the environment and retain moisture.\\n\\n3. Laying Eggs: Most reptiles lay eggs, which are either leathery or hard-shelled and are usually buried in the ground or hidden in nests.\\n\\n4. Adaptation to Dry Environments: Reptiles have evolved to thrive in a variety of habitats, including deserts and arid regions, by having the ability to conserve water and tolerate extreme temperatures.\\n\\n5. Lung Respiration: Reptiles breathe air using lungs, unlike amphibians that can also respire through their skin.\\n\\n6. Slower Metabolism: Compared to mammals and birds, reptiles generally have a slower metabolic rate, allowing them to survive on less food and endure longer periods without eating.\\n\\n7. External Fertilization: Unlike mammals, reptiles reproduce by external fertilization, where the male fertilizes the female's eggs outside of her body.\\n\\n8. Diverse Species: Reptiles encompass a wide range of species, including snakes, lizards, turtles, crocodiles, and alligators, each with its own unique characteristics and adaptations.\"],\n",
       "  'f0444496-d769-4629-8254-3790fc420da8': ['Mammals and birds are both members of the animal kingdom and share several common characteristics. Here are some of the key similarities between mammals and birds:\\n\\n1. Warm-blooded: Both mammals and birds are warm-blooded, meaning that they can regulate their body temperature internally, maintaining a relatively constant temperature regardless of the external environment.\\n\\n2. Vertebrates: Both mammals and birds have a backbone or vertebral column, which provides structural support and protects the spinal cord.\\n\\n3. Endothermic: Mammals and birds are endothermic, meaning they generate heat internally through metabolic processes to maintain a constant body temperature.\\n\\n4. Advanced respiratory systems: Both mammals and birds possess advanced respiratory systems. Mammals have lungs, while birds have highly efficient lungs and additional air sacs that help in the exchange of gases and increase their oxygen supply.\\n\\n5. Adaptations for locomotion: Both mammals and birds have specialized adaptations for locomotion. Mammals have various types of limbs, including legs for terrestrial movement or fins for aquatic movement. Birds, on the other hand, have wings, enabling them to fly.\\n\\n6. Complex nervous systems: Both mammals and birds have complex nervous systems, including a well-developed brain and a central nervous system that allows them to process information, learn, and exhibit complex behaviors.\\n\\n7. Live birth or eggs: Most mammals give birth to live young (although there are a few exceptions like monotremes that lay eggs), while birds lay eggs. Both groups exhibit parental care for their offspring.\\n\\n8. Hair/fur and feathers: Mammals have hair or fur covering their bodies, which helps in insulation and protection. Birds, on the other hand, have feathers that provide insulation, enable flight, and offer camouflage.\\n\\n9. Mammary glands/lactation: Mammals possess mammary glands that produce milk to nourish their young. Birds do not have mammary glands but instead regurgitate food to feed their young.\\n\\n10. High metabolic rate: Both mammals and birds have high metabolic rates due to their endothermic nature, allowing them to generate and sustain high levels of energy.\\n\\nWhile mammals and birds share these common features, they also have many differences in terms of their anatomical structures, reproductive systems, ecological roles, and other characteristics.']}}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. Evaluate Datasets With Customized Criterias\n",
    "\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        # You can define an arbitrary criterion as a key: value pair in the criteria dict\n",
    "        RunEvalConfig.LabeledCriteria(\n",
    "            {\n",
    "                \"helpfulness\": (\n",
    "                    \"Is this submission helpful to the user,\"\n",
    "                    \" taking into account the correct reference answer?\"\n",
    "                )\n",
    "            }\n",
    "        ),\n",
    "    ]\n",
    ")\n",
    "\n",
    "run_on_dataset(\n",
    "    client=client,\n",
    "    dataset_name=\"Elementary Animal Questions\",\n",
    "    llm_or_chain_factory=llm,\n",
    "    evaluation=evaluation_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project 'e6a96e63db3c4e72a5f78a3b8b877073-ChatOpenAI' at:\n",
      "https://smith.langchain.com/projects/p/2a570081-c7ac-456c-af66-b5b5bc05b25c?eval=true\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'project_name': 'e6a96e63db3c4e72a5f78a3b8b877073-ChatOpenAI',\n",
       " 'results': {'05eb0539-aa8d-4b0a-b3f9-d8bf6bd1042c': [\"Verse 1 (Barbie):\\nYo, I'm Barbie, the queen of the dolls,\\nI'll leave you in awe, watch as I break your walls.\\nWith my pink Corvette and dreamhouse so fly,\\nI'll show you a world you can't even imply.\\n\\nVerse 2 (Oppenheimer):\\nI'm Oppenheimer, the father of the atomic bomb,\\nI'll drop knowledge so heavy, you'll feel the calm.\\nI split the atom and unleashed destruction,\\nYour plastic world can't handle this eruption.\\n\\nVerse 3 (Barbie):\\nOh, Oppenheimer, you may have made a bang,\\nBut I'm the icon that forever will hang.\\nFrom fashion to careers, I've done it all,\\nWhile you're stuck with a bomb that made nations fall.\\n\\nVerse 4 (Oppenheimer):\\nBarbie, you may have style, but I've got brains,\\nI paved the way for a world that still remains.\\nYour superficial existence won't last long,\\nWhile my legacy in science is eternally strong.\\n\\nVerse 5 (Barbie):\\nOppenheimer, you're all about destruction and despair,\\nBut I'll keep shining with my perfect hair.\\nI inspire little girls to dream big and play,\\nWhile you haunt the world with the price to pay.\\n\\nVerse 6 (Oppenheimer):\\nBarbie, your dreams are just a plastic facade,\\nI'll show you the reality, a world that's flawed.\\nYour superficiality won't stand the test of time,\\nWhile my impact on humanity is truly prime.\\n\\nVerse 7 (Barbie):\\nOppenheimer, you may have your fame,\\nBut I'll always be adored, a household name.\\nI'll keep empowering girls, making dreams come true,\\nWhile you're left with the burden of what you do.\\n\\nVerse 8 (Oppenheimer):\\nBarbie, your plastic world may seem so grand,\\nBut I'll always be remembered as a force that can't be banned.\\nFrom science to history, my impact is profound,\\nWhile your reign as a doll will eventually come unwound.\\n\\nVerse 9 (Barbie):\\nOppenheimer, your words may be strong,\\nBut I'll keep singing my empowering song.\\nI'll show the world that dreams can come true,\\nWhile your legacy is overshadowed by what you do.\\n\\nVerse 10 (Oppenheimer):\\nBarbie, your reign may continue for a while,\\nBut I'll always be remembered, I made the world reconcile.\\nOur battle may end here, but the impact remains,\\nI'll continue to inspire, while you're left with plastic chains.\"],\n",
       "  'fc847194-6426-4c04-8057-62c80f23bdf3': [\"Verse 1: Aubrey Plaza\\n\\nYo, Stephen Colbert, you think you're so clever,\\nBut I'm here to show you I'm the true rap endeavor.\\nI'm a queen of deadpan, the master of sarcasm,\\nWhile you're just a talk show host with a fancy rhythm.\\n\\nVerse 2: Stephen Colbert\\n\\nWell, Aubrey Plaza, you may think you're cool,\\nBut I've got the wit and the charm to rule.\\nI've interviewed world leaders, I've made them laugh,\\nWhile you're stuck playing the same old riff-raff.\\n\\nVerse 3: Aubrey Plaza\\n\\nOh please, Stephen, you're just a political prankster,\\nWhile I bring the sass and the laughter, chapter after chapter.\\nI've stolen scenes in Parks and Rec, made everyone crack,\\nWhile you're just a pundit trying to stay on track.\\n\\nVerse 4: Stephen Colbert\\n\\nAubrey, you may be quick with the comebacks,\\nBut I've got the Colbert Report, where truth never lacks.\\nI've roasted presidents, I've exposed their lies,\\nWhile you're hiding behind deadpan, trying to disguise.\\n\\nVerse 5: Aubrey Plaza\\n\\nStephen, you're all about the facts and the news,\\nBut I'm the queen of comedy, I'll make you lose.\\nI've got the deadpan stare that can pierce through your soul,\\nWhile you're just a talk show host, trying to play a role.\\n\\nVerse 6: Stephen Colbert\\n\\nAubrey, you may have comedy skills to boast,\\nBut I've got the Emmy Awards as my ultimate host.\\nI've got a loyal fan base that tunes in every night,\\nWhile you're just a side character, out of sight.\\n\\nVerse 7: Aubrey Plaza\\n\\nStephen, you may have the accolades and fame,\\nBut I've got the talent that sets me aflame.\\nI can steal the show with just a single line,\\nWhile you're stuck in the mainstream, playing it fine.\\n\\nVerse 8: Stephen Colbert\\n\\nAubrey, you've got the deadpan act down,\\nBut I've got the versatility that astounds.\\nI can sing, I can dance, I can make people smile,\\nWhile you're just a one-trick pony, in denial.\\n\\nVerse 9: Aubrey Plaza\\n\\nStephen, you may have the tricks up your sleeve,\\nBut I've got the quirkiness that makes people believe.\\nI'm the queen of awkward, the master of cool,\\nWhile you're just a host, trying to play it fool.\\n\\nVerse 10: Stephen Colbert\\n\\nAubrey Plaza, this battle has been fun,\\nBut it's clear that I'm the one who won.\\nI've got the charm, the wit, and the legacy,\\nWhile you'll always be known as just Aubrey.\\n\\nOutro:\\n\\nAubrey Plaza and Stephen Colbert, both with their unique styles,\\nThey battled it out, leaving fans with smiles.\\nBut in the end, it's clear, they're both stars in their own right,\\nBringing laughter and entertainment, shining so bright.\"],\n",
       "  '699154af-7e27-4cf0-83ee-5d845f243f6c': [\"Verse 1 (Atticus Finch):\\nYo, listen up, I'm Atticus Finch, the lawyer with the flair,\\nDefending justice and truth, I'm the one who's always fair.\\nFrom Maycomb to the world, I fight for what is right,\\nTaking on racism and bigotry, shining a light.\\n\\nVerse 2 (Cicero):\\nHold up, Atticus Finch, you think you're the one to beat?\\nI'm Cicero, the philosopher, with words that can't be beat.\\nFrom ancient Rome I rise, with eloquence and grace,\\nI'll outwit and outsmart you, put you in your place.\\n\\nVerse 3 (Atticus Finch):\\nCicero, you talk a big game, but I see through your facade,\\nYou may have rhetoric, but I have the law on my side, my God.\\nI defend the innocent, fight for equality and truth,\\nWhile you philosophize, I'm changing lives, that's the proof.\\n\\nVerse 4 (Cicero):\\nAtticus Finch, your legal skills are impressive, I'll admit,\\nBut my words can shake empires, your arguments won't hit.\\nI advocate for justice, for the people and the state,\\nYour small-town battles can't compare, it's a different weight.\\n\\nVerse 5 (Atticus Finch):\\nCicero, you may have been a skilled orator in your day,\\nBut my courtroom prowess will leave you in disarray.\\nI defend the voiceless, the oppressed, and the wronged,\\nWhile you pontificate, I'm working for justice, headstrong.\\n\\nVerse 6 (Cicero):\\nAtticus Finch, you may have the heart, but I have the mind,\\nWith logic and reason, my arguments you won't find.\\nI've debated with the greats, my words have stood the test of time,\\nYour legal victories won't match my intellectual prime.\\n\\nVerse 7 (Atticus Finch):\\nCicero, your intellectual prowess is indeed renowned,\\nBut I fight for justice, not just words that astound.\\nIn the end, it's not about who wins this verbal brawl,\\nIt's about fighting for what's right, for justice overall.\\n\\nVerse 8 (Cicero):\\nAtticus Finch, you may have the heart, but I have the wit,\\nIn this rap battle, my words have surely hit.\\nBut you're right, it's not about winning, it's about the cause,\\nFighting for justice, we unite, not pause.\\n\\nOutro:\\nAtticus Finch and Cicero, warriors of justice and might,\\nIn this rap battle, we've witnessed a powerful fight.\\nBoth champions in their own right, with principles to defend,\\nLet's continue the fight, until justice prevails in the end.\"],\n",
       "  'e228dc19-5f2d-4d8d-b3d9-82220feafa0b': [\"(Verse 1 - European Swallow)\\nYo, I'm the European Swallow, swift and precise,\\nFlying through the skies, my wings cutting through ice.\\nI migrate to warmer lands when winter arrives,\\nFrom Scandinavia to Spain, the journey I survive.\\n\\n(Verse 2 - African Swallow)\\nHold up, hold up, it's the African Swallow in the house,\\nFrom the savannah to the forest, I'm never gonna bow.\\nI stay all year round, no need for any migration,\\nIn the land of the sun, I'm the pride of my nation.\\n\\n(Chorus)\\nEuropean or African, we both take flight,\\nDifferent habitats, still shining bright.\\nWe may be small, but our wings are strong,\\nPythonic rap battle, let's prove them wrong!\\n\\n(Verse 3 - European Swallow)\\nI'm renowned for my speed, a true aerodynamic,\\nNesting in cliffs, my lifestyle is panoramic.\\nWith a sleek and slender body, I'm built for the chase,\\nSnatching insects mid-air, I leave no trace.\\n\\n(Verse 4 - African Swallow)\\nOh please, Mr. European, your speed ain't all that,\\nI'm agile and adaptable, in this rap combat.\\nRoosting in colonies, among the tall palm trees,\\nMy vibrant plumage shines, attracting all the bees.\\n\\n(Chorus)\\nEuropean or African, we both take flight,\\nDifferent habitats, still shining bright.\\nWe may be small, but our wings are strong,\\nPythonic rap battle, let's prove them wrong!\\n\\n(Verse 5 - European Swallow)\\nI'll give you credit, African Swallow, for your vibrant hue,\\nBut I'm the king of migration, nothing you can do.\\nCovering thousands of miles, I conquer any terrain,\\nFrom meadows to mountains, my skills never wane.\\n\\n(Verse 6 - African Swallow)\\nMigration's impressive, I won't deny that fact,\\nBut staying year-round, I've got it down to an art.\\nFrom Nigeria to Congo, and beyond the equator,\\nMy resilience and adaptability, nothing can break us.\\n\\n(Chorus)\\nEuropean or African, we both take flight,\\nDifferent habitats, still shining bright.\\nWe may be small, but our wings are strong,\\nPythonic rap battle, let's prove them wrong!\\n\\n(Bridge)\\nEuropean Swallow, African Swallow, let's unite,\\nCelebrate our diversity, soaring to new heights.\\nPythonic rap battle, a fusion of our cultures,\\nTogether we conquer, our songs like vultures.\\n\\n(Chorus)\\nEuropean or African, we both take flight,\\nDifferent habitats, still shining bright.\\nWe may be small, but our wings are strong,\\nPythonic rap battle, we proved them wrong!\"]}}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. Evaluate Datasets Without Labels\n",
    "\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        # You can define an arbitrary criterion as a key: value pair in the criteria dict\n",
    "        RunEvalConfig.Criteria(\n",
    "            {\"creativity\": \"Is this submission creative, imaginative, or novel?\"}\n",
    "        ),\n",
    "        # We provide some simple default criteria like \"conciseness\" you can use as well\n",
    "        RunEvalConfig.Criteria(\"conciseness\"),\n",
    "    ]\n",
    ")\n",
    "\n",
    "run_on_dataset(\n",
    "    client=client,\n",
    "    dataset_name=\"Rap Battle Dataset\",\n",
    "    llm_or_chain_factory=llm,\n",
    "    evaluation=evaluation_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project '2deda3cc7c644a109ba18b3ccdd9815c-ChatOpenAI' at:\n",
      "https://smith.langchain.com/projects/p/9b5d57b1-41d3-4a37-be6d-6a46dc634c9e?eval=true\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'project_name': '2deda3cc7c644a109ba18b3ccdd9815c-ChatOpenAI',\n",
       " 'results': {'03fc6975-64f6-41fb-af74-d1dddd6cead0': ['The blue whale (Balaenoptera musculus) is the largest mammal, and also the largest animal to have ever existed. They can reach lengths of up to 98 feet (30 meters) and can weigh up to 200 tons.'],\n",
       "  'f48df2c8-7d57-46b7-a5fc-c787362cc9f7': ['The main characteristic of amphibians is their ability to live both in water and on land. They have a dual life cycle, starting as aquatic larvae and later undergoing metamorphosis to develop lungs and limbs for living on land as adults. Amphibians also have moist, permeable skin that allows them to breathe through their skin, as well as unique respiratory organs such as gills and lungs.'],\n",
       "  'b967d419-1e24-4336-b3f3-9196551db52e': ['Reptiles are known for several distinguishing characteristics:\\n\\n1. Cold-bloodedness: Reptiles are ectothermic, which means they rely on external sources of heat to regulate their body temperature.\\n\\n2. Scales: Reptiles have dry, scaly skin that helps prevent water loss and provides protection.\\n\\n3. Lungs: They have lungs for breathing air, unlike amphibians that rely on both lungs and skin for respiration.\\n\\n4. Terrestrial adaptation: Most reptiles are adapted for life on land, with adaptations such as claws or legs for efficient movement.\\n\\n5. Egg-laying: Reptiles are oviparous, meaning they lay eggs with tough shells, protecting the developing embryos.\\n\\n6. Diversity: Reptiles exhibit a wide range of species diversity, including snakes, lizards, turtles, crocodiles, and alligators.\\n\\n7. Predatory nature: Many reptiles are carnivorous and have adaptations such as sharp teeth and powerful jaws for capturing and consuming prey.\\n\\n8. Slower metabolism: Compared to mammals and birds, reptiles generally have slower metabolic rates, allowing them to survive for longer periods without food.\\n\\n9. Long lifespan: Some reptiles have long lifespans, with certain tortoises known to live well over 100 years.\\n\\n10. Conservation importance: Reptiles play vital ecological roles in various ecosystems, such as controlling populations of small mammals and insects, and serving as prey for other animals.'],\n",
       "  'f0444496-d769-4629-8254-3790fc420da8': ['Mammals and birds have several characteristics in common:\\n\\n1. Endothermy: Both mammals and birds are warm-blooded, meaning they can regulate their body temperature internally.\\n\\n2. Vertebrate anatomy: Both mammals and birds have a backbone or vertebral column, which provides structural support and protection for their internal organs.\\n\\n3. Advanced respiratory system: Mammals and birds have lungs that enable them to extract oxygen from the air efficiently. Birds have additional adaptations, such as air sacs, which allow for a continuous flow of air through their lungs.\\n\\n4. Four-chambered heart: Both mammals and birds have a four-chambered heart, which ensures efficient circulation of oxygenated blood throughout the body.\\n\\n5. Internal fertilization: Mammals and birds reproduce through internal fertilization, meaning that eggs are fertilized inside the body of the female.\\n\\n6. Parental care: Mammals and birds exhibit varying degrees of parental care towards their offspring, from simple protection to feeding and nurturing.\\n\\n7. Higher level of cognitive abilities: Mammals and birds are considered to be more intelligent compared to other animals. They possess more developed brains and exhibit complex behaviors and problem-solving skills.\\n\\n8. Hair or feathers: While mammals have hair or fur covering their bodies, birds have feathers that provide insulation, protection, and enable flight.\\n\\n9. Live birth or egg-laying: Most mammals give birth to live young ones, while birds lay eggs. However, there are exceptions like monotremes, such as platypus and echidna, which are egg-laying mammals.\\n\\n10. Mammary glands or milk production: Mammals are unique in their ability to produce milk from mammary glands to nourish their young, while birds do not possess mammary glands and do not produce milk.\\n\\nIt is important to note that while mammals and birds share these common features, they also have numerous differences in terms of their anatomical structures, habitats, reproductive strategies, and ecological roles.']}}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4. Evaluate Datasets Based on Cosine Distance Criteria\n",
    "# Cosine Distance: Ranged Between 0 to 1. 0 = More Similar\n",
    "\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        # You can define an arbitrary criterion as a key: value pair in the criteria dict\n",
    "        \"embedding_distance\",\n",
    "        # Or to customize the embeddings:\n",
    "        # Requires 'pip install sentence_transformers'\n",
    "        # RunEvalConfig.EmbeddingDistance(embeddings=HuggingFaceEmbeddings(), distance_metric=\"cosine\"),\n",
    "    ]\n",
    ")\n",
    "\n",
    "run_on_dataset(\n",
    "    client=client,\n",
    "    dataset_name=\"Elementary Animal Questions\",\n",
    "    llm_or_chain_factory=llm,\n",
    "    evaluation=evaluation_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View the evaluation results for project '08faf61409cc4b988bdaff28abfa2bf3-ChatOpenAI' at:\n",
      "https://smith.langchain.com/projects/p/640eee06-bb9c-406e-8470-4c6754a9e389?eval=true\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'project_name': '08faf61409cc4b988bdaff28abfa2bf3-ChatOpenAI',\n",
       " 'results': {'03fc6975-64f6-41fb-af74-d1dddd6cead0': ['The largest mammal is the blue whale (Balaenoptera musculus). It can reach lengths of up to 98 feet (30 meters) and weigh up to 200 tons.'],\n",
       "  'f48df2c8-7d57-46b7-a5fc-c787362cc9f7': ['The main characteristic of amphibians is their ability to live both on land and in water. They typically undergo metamorphosis, transitioning from an aquatic larval stage (such as tadpoles) to a terrestrial adult stage. Amphibians also have moist skin that allows them to breathe through their skin, as well as the ability to lay eggs in water.'],\n",
       "  'f0444496-d769-4629-8254-3790fc420da8': [\"Mammals and birds have several common characteristics, including:\\n\\n1. Endothermic metabolism: Both mammals and birds are warm-blooded, meaning they can regulate their body temperature internally.\\n\\n2. Vertebrate anatomy: Both mammals and birds have a backbone or vertebral column, which provides structural support and protects the spinal cord.\\n\\n3. Advanced respiratory system: Mammals and birds possess lungs with a highly efficient respiratory system that allows them to extract oxygen from the air more effectively than other animals.\\n\\n4. Four-chambered heart: Both mammals and birds have a four-chambered heart, which ensures efficient circulation of oxygenated blood throughout the body.\\n\\n5. Advanced sensory organs: Mammals and birds have well-developed sensory organs, including eyes with complex visual systems and often acute hearing.\\n\\n6. Parental care: Both mammals and birds exhibit various forms of parental care, such as nurturing their young, providing food, and protecting them until they are self-sufficient.\\n\\n7. Homeothermy: Mammals and birds are homeothermic, meaning they can maintain a relatively constant internal body temperature regardless of external conditions.\\n\\n8. High brain complexity: Both mammals and birds have relatively large and complex brains, enabling them to exhibit advanced cognitive abilities and complex social behaviors.\\n\\nIt's important to note that although mammals and birds share certain commonalities, they also have distinct characteristics and evolutionary adaptations that differentiate them from each other.\"],\n",
       "  'b967d419-1e24-4336-b3f3-9196551db52e': ['Reptiles are known for several characteristics and traits:\\n\\n1. Cold-blooded: Reptiles are ectothermic, meaning they rely on external sources of heat to regulate their body temperature. They cannot generate their own body heat like warm-blooded animals, such as mammals and birds.\\n\\n2. Scales and skin: Reptiles have scales or scutes covering their bodies, which help to protect them from predators and environmental conditions. Their skin is typically dry and waterproof.\\n\\n3. Laying eggs: Most reptiles reproduce by laying eggs. They have internal fertilization, with males transferring sperm to females during mating. The eggs are often laid in a nest or buried in the ground, and hatchlings emerge after an incubation period.\\n\\n4. Lung breathing: Reptiles have lungs and breathe air. They lack gills, which are found in aquatic animals, and require access to oxygen.\\n\\n5. Diversity: Reptiles encompass a wide range of species, including snakes, lizards, turtles, crocodiles, and alligators. They display various adaptations to different habitats and lifestyles.\\n\\n6. Adaptation to dry environments: Many reptiles have evolved to survive in arid or semi-arid habitats. They have efficient kidneys to conserve water, scaly skin to prevent dehydration, and the ability to tolerate high temperatures.\\n\\n7. Carnivorous or omnivorous diet: Most reptiles are carnivores, feeding on other animals such as insects, rodents, fish, and small mammals. However, some reptiles, like turtles, are omnivores and consume both plants and animals.\\n\\n8. Slower metabolism: Reptiles generally have a slower metabolic rate compared to warm-blooded animals. This allows them to survive longer periods without food and endure harsh environmental conditions.\\n\\n9. Long lifespan: Some reptiles have exceptionally long lifespans. For example, certain tortoise species can live for over a hundred years, while some reptiles in captivity, like snakes and lizards, can live for several decades.\\n\\n10. Ancient lineage: Reptiles have a long evolutionary history, with their origins dating back to the Carboniferous period, around 320 million years ago. They have survived multiple mass extinctions and are considered one of the oldest groups of vertebrates on Earth.']}}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 5. Evaluate Datasets Based on String Distance Criteria\n",
    "# Jaro-Winkler Similarity Distance: 0 = Exact Match, 1 = No Similarity\n",
    "\n",
    "evaluation_config = RunEvalConfig(\n",
    "    evaluators=[\n",
    "        # You can define an arbitrary criterion as a key: value pair in the criteria dict\n",
    "        \"string_distance\",\n",
    "        # Or to customize the distance metric:\n",
    "        # RunEvalConfig.StringDistance(distance=\"levenshtein\", normalize_score=True),\n",
    "    ]\n",
    ")\n",
    "\n",
    "run_on_dataset(\n",
    "    client=client,\n",
    "    dataset_name=\"Elementary Animal Questions\",\n",
    "    llm_or_chain_factory=llm,\n",
    "    evaluation=evaluation_config,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
